package com.spark.sql

import org.apache.spark.sql.SparkSession

/**
  * RDD转换为DataFrame
  */
object RDD2DataFrameApp {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().appName("RDD2DF").master("local[2]").getOrCreate()

    // load text file
    val rdd = spark.sparkContext.textFile("file:///Users/username/workspace_code/learn/spark-learn/datasets/infos.txt")
    val infoRdd = rdd.map(_.split(",")).map(line=>Info(line(0).toInt, line(1), line(2).toInt, line(3).toInt))

    // 导入隐式转换，然后调用toDF将RDD转换为DataFrame
    import spark.implicits._
    val infoDF = infoRdd.toDF()
    infoDF.show()

    // 使用DF过滤
    val filterDF = infoDF.filter(infoDF.col("age") > 30)
    filterDF.show()

    // 将DataFrame处理成表的形式处理
    infoDF.createOrReplaceTempView("infos")
    val sqlDF = spark.sql("select * from infos where age > 30")
    sqlDF.show()

    spark.stop()

  }
  case class Info(id: Int, name: String, age: Int, height: Int)
}
